{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tensorflow2.0 小练习"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "@tf.function\n",
    "def func(x):\n",
    "    x_exp = tf.math.exp(x)\n",
    "    print(x_exp)\n",
    "    out = x_exp+1\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "func(tf.constant(0.1, dtype=tf.float32))\n",
    "func(tf.constant(1., dtype=tf.float32))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 能用到的API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "tf.math.exp()\n",
    "tf.math.log()\n",
    "tf.reduce_sum()\n",
    "tf.reduce_mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 实现softmax函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([ 2.7182817  7.389056  20.085537  54.59815  ], shape=(4,), dtype=float32)\n",
      "[ 2.71828183  7.3890561  20.08553692 54.59815003]\n",
      "Tensor(\"Exp:0\", shape=(4,), dtype=float32)\n",
      "[[ 0.56577835  0.03925728  0.12253202  0.42823133 -0.30519943]\n",
      " [-0.62723232 -0.77811349 -0.51248257 -0.08009214  0.45644417]\n",
      " [-0.47520551  1.20698361  0.35421463  1.51033826  0.31313441]\n",
      " [ 0.77509969  0.17709761  0.51255336  0.70945003  0.58216116]\n",
      " [ 0.18728327 -0.60192896  1.2517836   0.09139799 -0.43010604]\n",
      " [-0.49701167  1.5995401  -0.69059412 -0.06869925 -3.53611644]\n",
      " [-1.13273144  0.85586692  0.11808564 -1.48678667  1.77579503]\n",
      " [ 0.18899566  1.09708078 -0.07208181 -1.89496327  1.29759657]\n",
      " [-1.25227742 -0.54591516 -0.44185156 -0.51002137 -0.5215976 ]\n",
      " [ 0.7112895  -1.19476881  0.26648144  0.62118595 -1.96629359]]\n",
      "----------------------------------------\n",
      "tf.Tensor(\n",
      "[[ 0.85059956]\n",
      " [-1.54147636]\n",
      " [ 2.90946539]\n",
      " [ 2.75636184]\n",
      " [ 0.49842986]\n",
      " [-3.19288139]\n",
      " [ 0.13022949]\n",
      " [ 0.61662792]\n",
      " [-3.27166311]\n",
      " [-1.56210551]], shape=(10, 1), dtype=float64)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "print(tf.math.exp([1, 2, 3, 4.]))\n",
    "print(np.exp([1, 2, 3, 4.]))\n",
    "\n",
    "@tf.function\n",
    "def func():\n",
    "    print(tf.math.exp([1, 2, 3, 4.]))\n",
    "func()\n",
    "\n",
    "test_data = np.random.normal(size=[10, 5])\n",
    "print(test_data)\n",
    "print('--'*20)\n",
    "print(tf.reduce_sum(test_data, axis=1, keepdims=True)) # [N, 1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# softmax function: $\\frac{e^{x_i}}{\\sum_{j=1}^{d}{e^{x_j}}}$\n",
    "# $\\sum_{i}{\\frac{e^{x_i}}{\\sum_{j=1}^{d}{e^{x_j}}}} == 1$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ True,  True,  True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def softmax(x):\n",
    "    epsilon = 1e-12\n",
    "    '''shape of x [N, d]'''\n",
    "    x_exp = tf.math.exp(x) # shape=[N, d]\n",
    "    denominator = tf.reduce_sum(x_exp, axis=1, keepdims=True) #shape=[N, 1]\n",
    "    prob_x = x_exp/(denominator+epsilon)\n",
    "    return prob_x\n",
    "\n",
    "test_data = np.random.normal(size=[10, 5])\n",
    "(softmax(test_data).numpy() - tf.nn.softmax(test_data, axis=-1).numpy())**2 <0.000001"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 实现sigmoid函数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# $\\frac{1}{1+e^{-x}}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ True,  True,  True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def sigmoid(x):\n",
    "    '''shape=[N, d]'''\n",
    "    x_exp = tf.math.exp(-x)\n",
    "#     prob_x = 1/(1+x_exp)\n",
    "    prob_x = tf.divide(1, tf.add(1, x_exp))\n",
    "    return prob_x\n",
    "\n",
    "test_data = np.random.normal(size=[10, 5])\n",
    "(sigmoid(test_data).numpy() - tf.nn.sigmoid(test_data).numpy())**2 < 0.0001"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 实现 softmax 交叉熵loss函数\n",
    "# $\\sum_{c}{-p(y=c|x)*log(p(\\hat{y}=c|x))}$\n",
    "# $s_{ij} = a_{ij}*b_{ij}$ \n",
    "# $s_{ik} = \\sum_j{a_{ij}*b_{jk}}$ matmul"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def softmax_ce(p, label):\n",
    "    '''\n",
    "        x shape = [N, c] ===> probability\n",
    "        label shape = [N, c]\n",
    "    '''\n",
    "    loss = tf.reduce_mean(tf.reduce_sum(-label*tf.math.log(p), axis=1))\n",
    "    return loss\n",
    "\n",
    "test_data = np.random.normal(size=[10, 5])\n",
    "prob = tf.nn.softmax(test_data)\n",
    "label = np.zeros_like(test_data)\n",
    "label[np.arange(10), np.random.randint(0, 5, size=10)]=1.\n",
    "\n",
    "((tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(label, test_data))\n",
    "  - softmax_ce(prob, label))**2 < 0.0001).numpy()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 实现 sigmoid 交叉熵loss函数\n",
    "\n",
    "# $-p(y=1|x)*log(p(\\hat{y}=1|x)-p(y=0|x)*log(p(\\hat{y}=0|x))$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0824 13:53:24.909731 4684809664 deprecation.py:323] From /Users/jerrik/a3/envs/tf2.0/lib/python3.7/site-packages/tensorflow/python/ops/nn_impl.py:182: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1. 1. 0. 1. 1. 1. 1. 1. 1. 1.]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def sigmoid_ce(p, label):\n",
    "    epsilon = 1e-12\n",
    "    '''shape = [N,]'''\n",
    "    losses = -label * tf.math.log(p+epsilon) - (1.-label) * tf.math.log(1.-p+epsilon) # shape=[N,]\n",
    "    loss = tf.reduce_mean(losses)\n",
    "    return loss\n",
    "\n",
    "test_data = np.random.normal(size=[10])\n",
    "prob = tf.nn.sigmoid(test_data)\n",
    "label = np.random.randint(0, 2, 10).astype(test_data.dtype)\n",
    "print (label)\n",
    "\n",
    "((tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(label, test_data))- sigmoid_ce(prob, label))**2 < 0.0001).numpy()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(1.105171, shape=(), dtype=float32)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=1395, shape=(), dtype=float32, numpy=2.105171>"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# @tf.function\n",
    "def func(x):\n",
    "    x_exp = tf.math.exp(x)\n",
    "    out = x_exp+1\n",
    "    return out\n",
    "\n",
    "func(0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 2.7182817,  7.389056 , 20.085537 , 54.59815  ], dtype=float32)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
